#!/usr/bin/env python
#_*_encoding:utf-8_*_

import os, sys, traceback
import fnmatch
import string,time,shutil
import ConfigParser,getopt
import datetime

try:
    from hashlib import md5 as md5instance
except:
    from md5 import new as md5instance
from FCLog import FCLog

_program_         = 'StatLogTask.py'
_version_         = '1.00.00'
_lastmodifytime_  = '2010-09-11'

CONF_PATH         = "../etc/StatLogTask.conf"
STAT_SUCCESS_LOG  = "../log/stat_success.log"
STAT_FAIL_LOG     = "../log/stat_fail.log"


def    PrintVersion():
        print """
+---------------------------------------------------------------+
 Name:%s                                                       
 Version:%s                                                    
 LastChangeTime:%s                                             
 Author:ghu                                                  
+---------------------------------------------------------------+""" % (_program_, _version_, _lastmodifytime_)


def    PrintUsage():
        print """
+---------------------------------------------------------------+
       Usage: %s                                               
       -v  --version  :output version information and exit. 
       -p  --path     :deal the file in path and exit,you must input a path.
                       Then ,you must input a date: "-d date".
       -d  --date     :input just as "YYYYmmdd".
                       deal the file with the date and exit.
                       You must input a path : "-p path".
       -h  --help     :display this help and exit.                      
+---------------------------------------------------------------+""" % _program_


def    getOption():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "vhp:d:", ["help", "version","path=","date="])
    except StandardError, e:
        print e
        PrintUsage()
        sys.exit(2)
    
    if len(opts) == 0:
    #没有命令参数
        return
    else:
    #有命令参数，需要执行
        srcpath  = ''
        dealdate = ''
        for o,a in opts:
            if o in ("-h", "--help"):
                PrintUsage()
                sys.exit()
            if o in ("-v", "--version"):
                PrintVersion()
                sys.exit()
            if o in ("-p","--path"):
                srcpath = a
            if o in ("-d","--date"):
                dealdate = a
        #此时情况为手动操作，必须同时存在时间和目录
        if srcpath == '':
            print "You must input a path where has former log."
            PrintUsage()
            sys.exit()
        if dealdate == '' or len(dealdate) != 8 :
            print "You must input a right date."
            PrintUsage()
            sys.exit()
        else:
            Handdeal(srcpath,dealdate)
            sys.exit()
      
#存放读取配置文件中各个模块的信息        
class StatInfo:
    def __init__(self):
        self.taskname         = ''
        self.statsrcpath      = ''
        self.statwritemodule  = ''
        self.bcpformat        = ''
        self.statdstpath      = '' 
        self.stattmppath      = ''
        self.suclog           = None
        self.errlog           = None
        

"""
GatherMap  = {'UPAREAID':'', 'PROTYPEDESC':'', 'DATASOURCE':'', 'DATADATE':'', 'STATTIME':0, 'DATAFROM':'' ,\
             'DATADEST':'', 'FILETYPE':'', 'SUCCFILENUM':0, 'SUCCFILESIZE':0, 'ERRFILENUM':0, 'ERRFILESIZE':0, 'SUCCRECORDNUM':0, 'ERRRECORDNUM':0 ,\
             'ENTRYFILENUM':0, 'ENTRYFILESIZE':0, 'SENDENTRYNUM':0, 'SENDENTRYSIZE':0, 'STATPROGRAM':0}  
"""

def     WriteStatLog(filename,stattaskinfo,gatherfilefullname,timeforname):
    #汇总日志固定格式
    Gatherkeyname = ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE', 'STATTIME', 'DATAFROM',\
                          'DATADEST', 'FILETYPE', 'SUCCFILENUM', 'SUCCFILESIZE', 'ERRFILENUM', 'ERRFILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM',\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE', 'STATPROGRAM']  
    #备份原始日志所有关键字 
    Formerkeyname = ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE', 'DEALDATE', 'DATAFROM',\
                          'DATADEST', 'FILENAME','FILETYPE', 'FILESIZE', 'STATUS', 'SUCCRECORDNUM', 'ERRRECORDNUM',\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE', 'STATPROGRAM']     
    try:
        
        fileInput = open(filename, 'r')                
        lines = fileInput.readlines()
        statdstlines   = []             #用于存放需要写统计日志的记录
        gatherdstlines = []             #用于存放需要写汇总日志的记录
        
        records_map    = {}             #存放MD5值：汇总日志的字段值
        sCount         = 0              #写统计日志的成功记录数
        eCount         = 0              #写统计日志的失败记录数
        
        statbcplist = stattaskinfo.bcpformat.split('\\t')  #将配置文件中设定的bcp格式切分
        
        
        #统计日志bcp文件命名规范，根据数据库的要求修改
        statbcpname = 'Stat_' + stattaskinfo.taskname + '_' + timeforname + '_' + str(time.time()) + '.bcp'
        
        #统计日志目的目录全路径和临时目录全路径
        dstfullname = os.path.join(stattaskinfo.statdstpath,statbcpname)
        tmpfullname = os.path.join(stattaskinfo.stattmppath,statbcpname)

        for line in lines:
            try:
                line = line[:-1]
                Former_Map  = {}
           #将一条记录的信息以键对值读入Former_Map中,供写统计日志和写汇总日日志引用
           
                Former_Keys = line.split('\t')  #得到键值对列表
           
                for key in Former_Keys[1:]:
                    former_key = key.split(':')   #将键值对切分
                    formerbig_key = former_key[0].upper()
                
                    if formerbig_key in ['DEALTIME', 'FILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM' ,\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE']:
                        if former_key[1] == '' :
				Former_Map[formerbig_key] = 0
			else :
				Former_Map[formerbig_key] = int(former_key[1])
                    else:
                        Former_Map[formerbig_key] = str(former_key[1])
            
            #检查记录行是否符合规范，主要检查FILENAME键值对是否存在和STATUS是否等于0或者-1。
                if  not Former_Map.has_key('FILETYPE')  :
                    stattaskinfo.errlog.error("[%s]formerlog line criterion error,no FILETYPE in %s line: %s "%(stattaskinfo.taskname,filename,line))
                    continue
                elif  not Former_Map.has_key('FILENAME')  :
                    stattaskinfo.errlog.error("[%s]formerlog line criterion error,no FILENAME in %s line: %s "%(stattaskinfo.taskname,filename,line))
                    continue
                elif  not Former_Map.has_key('STATUS') :
                    stattaskinfo.errlog.error("[%s]formerlog line criterion error,no STATUS in %s line: %s "%(stattaskinfo.taskname,filename,line))
                    continue
                elif Former_Map['FILENAME'] == '':
                    stattaskinfo.errlog.error("[%s]formerlog line criterion FILENAME error,in %s line: %s "%(stattaskinfo.taskname,filename,line))
                    continue
                elif Former_Map['STATUS'] != '-1' and Former_Map['STATUS'] != '0':
                    stattaskinfo.errlog.error("[%s]formerlog line criterion STATUS error,in %s line: %s "%(stattaskinfo.taskname,filename,line))
                    continue
                else:
                
#                   print "转成统计日志的记录格式"
                    #转成统计日志的记录格式       
                    if stattaskinfo.statwritemodule != '0' :
#                       "需要写 statlog"
                        programlist = stattaskinfo.statwritemodule.split(',')
#                       print programlist
                    
                        if Former_Map.has_key('STATPROGRAM'):
                           
                            if Former_Map['STATPROGRAM'] in programlist :
#                               "原始备份日志中有模块字段"                          
                                statdstline = ''
                                writebcpstat = []
                                for i in range(0,len(statbcplist)) :
                                    stat_bigbcp = statbcplist[i].upper()
                                    if  Former_Map.has_key(stat_bigbcp):
#                                       print Former_Map[stat_bigbcp]
#                          需要根据数据库给的bcp格式替换成值
                                        writebcpstat.append(Former_Map[stat_bigbcp])
                                    else:
                                        if stat_bigbcp in Formerkeyname:
                                            writebcpstat.append('')
                                        else:
                                            writebcpstat.append(statbcplist[i])
                                for i in range(0,len(writebcpstat)) :
                                    statdstline = statdstline + str(writebcpstat[i]) + "\t"
#                               print statdstline
                                if statdstline:
                                    statdstlines.append(statdstline[:-1] + "\n")
                                    sCount += 1
                                else:
                                    stattaskinfo.errlog.error("[%s]write statlog in %s ,fail to deal line: %s "%(stattaskinfo.taskname,filename,line))
                                    eCount += 1
  #                     else :
#                           print "行信息模块字段不在配置文件需要写统计日志的模块中"
                    
                    #汇总日志关键字，用来替换成值。形成一个值列表
                    Gatherkeys = ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE', 'STATTIME', 'DATAFROM',\
                          'DATADEST', 'FILETYPE', 'SUCCFILENUM', 'SUCCFILESIZE', 'ERRFILENUM', 'ERRFILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM',\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE', 'STATPROGRAM']
                    Gatherkeys[4] = int(time.time())     #更新合并后的最新操作时间STATTIME
                    if  Former_Map['STATUS'] == '0':    #处理错误的记录
                        Gatherkeys[8]  = 1
                        if Former_Map.has_key('FILESIZE'):
                            Gatherkeys[9]  = Former_Map['FILESIZE']
                        else :                           
                            Gatherkeys[9]  = 0
                        Gatherkeys[10] = 0
                        Gatherkeys[11] = 0
                    else :                                #处理正确的记录
                        Gatherkeys[8]  = 0
                        Gatherkeys[9]  = 0
                        Gatherkeys[10] = 1
                        if Former_Map.has_key('FILESIZE'):
                            Gatherkeys[11]  = Former_Map['FILESIZE']
                        else :
                            Gatherkeys[11]  = 0          
                    for i in range(0,len(Gatherkeys)):
                        #将统计日志Gatherkeys的其它关键字替换成值
                        if  not  Former_Map.has_key(Gatherkeys[i]):
                            if Gatherkeys[i] in [ 'SUCCRECORDNUM', 'ERRRECORDNUM' ,'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE']:
                                Gatherkeys[i] = 0
                            elif Gatherkeys[i] in ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE','DATAFROM','DATADEST','STATPROGRAM'] :
                                Gatherkeys[i] = ''
                        else :
                            Gatherkeys[i] = Former_Map[Gatherkeys[i]]

                
                    checkstr = Gatherkeys[0] + Gatherkeys[1] + Gatherkeys[2] + Gatherkeys[3] + Gatherkeys[5] + Gatherkeys[6] + Gatherkeys[7] + Gatherkeys[18]
                
                    #对以上ckeckstr做MD5值校验，汇总信息           
                    m = md5instance()
                    m.update(checkstr)
                    md5_value = m.hexdigest()
                    if not records_map.has_key(md5_value):
                        records_map[md5_value] = Gatherkeys
                    else :
                        records_map[md5_value][4] = int(time.time())
                        for i in range(8,18):
                            records_map[md5_value][i] += Gatherkeys[i]
            except:
		print "----->", formerbig_key, "<----"
		raise
                stattaskinfo.errlog.error("[%s]formerlog line deal error,in %s  line: %s "%(stattaskinfo.taskname,filename,line))
        fileInput.close()
            
        #将汇总信息转成键值对形式的记录
        for md5_key in  records_map :
            gatherdstline = ''
            for i in range(0,len(Gatherkeyname)):
                gatherdstline = gatherdstline + str(Gatherkeyname[i]) + ":" +  str(records_map[md5_key][i]) + "\t" 
            gatherdstlines.append(gatherdstline[:-1] + '\n')
        
        #写统计日志
        if sCount > 0:           
            fileOutput = open(tmpfullname,'w')
            fileOutput.writelines(statdstlines)
            fileOutput.close()
            shutil.move(tmpfullname, dstfullname)
            stattaskinfo.suclog.info("[%s]write statlog  success. Formerlog is %s.Success total linenum: %d    Error total linenum: %d" %(stattaskinfo.taskname,filename, sCount, eCount))            
        elif eCount > 0:
            stattaskinfo.suclog.info("[%s]write statlog  error. Formerlog is %s." %(stattaskinfo.taskname,filename))
        
        #写汇总日志
        if len(gatherdstlines) > 0:
            try:
                fileOutput = open(gatherfilefullname,'a')
                fileOutput.writelines(gatherdstlines)
                fileOutput.close()
                stattaskinfo.suclog.info("[%s]write gatherlog  success. Formerlog is %s."%(stattaskinfo.taskname,filename))
            except:
                stattaskinfo.errlog.log_traceback()
                stattaskinfo.errlog.error("[%s]write gatherlog  error. Formerlog is %s."%(stattaskinfo.taskname,filename))
        else:            
            stattaskinfo.errlog.error("[%s]write gatherlog  error. Formerlog is %s."%(stattaskinfo.taskname,filename))
   
        
    except:
        stattaskinfo.errlog.log_traceback()
        stattaskinfo.errlog.error("[%s]write statlog and gatherlog  error. Formerlog is %s."%(stattaskinfo.taskname,filename))                                            
 

def     DealStatInfo(upareaid = '',protypedesc = '',datasource = '',datafrom = '',datadest = '',filetype = '',frontdate = '',statprogram = '',enddate = '',statlogpath = ''):
    try :
        get_key = ['upareaid','protypedesc','datasource','datafrom','datadest','filetype','statprogram']
        get_value = []
        check_keys = []
        
        dstlinelist = []
        #根据参数值添加
        get_value.append(upareaid)
        get_value.append(protypedesc)
        get_value.append(datasource)
        get_value.append(datafrom)
        get_value.append(datadest)
        get_value.append(filetype)
        get_value.append(statprogram)
        
        #将函数参数及其值以键值对的形式存放于check_key列表里面，没有赋值的参数不存放
        for i in range(0,len(get_value)):
            keystr = ''
            if get_value[i] != '' :
                keystr = str(get_key[i].upper()) + ':' + str(get_value[i])
                check_keys.append(keystr)
 
#       print check_keys   
        #检查时间和汇总日志存放的目录，取汇总日志文件
        if frontdate == '' or enddate == '' :
#       "您没有同时输入开始时间和结束时间"
            return []
        elif len(frontdate) != 10 or len(enddate) != 10:
#       "您输入的时间有误"
            return []
        elif int(frontdate) > int(enddate):
#       "前后时间冲突,开始时间大于结束时间"
            return []
        if statlogpath == '':
#       "您没有指定存放汇总日志的目录statlogpath"
            return []
        elif  not os.path.exists(statlogpath) :
#       "您输入的日志目录不存在"
            return []
        else :
            
            yeare,mone,daye,houre  = int(enddate[0:4]),int(enddate[4:6]),int(enddate[6:8]),int(enddate[8:10])
            try:
                check_enddate      = datetime.datetime(yeare,mone,daye,houre)
            except:
#           "The end time you inputted is wrong"
                return []
            
            years,mons,days,hours  = int(frontdate[0:4]),int(frontdate[4:6]),int(frontdate[6:8]),int(frontdate[8:10])
            try:
                check_frontdate      = datetime.datetime(years,mons,days,hours)
            except:
#           "The start time you inputted is wrong"
                return []   
  
#           dstnum_suc = 0
            while (int(frontdate) <= int(enddate)):
                try:
                    year,mon,day,hour  = int(frontdate[0:4]),int(frontdate[4:6]),int(frontdate[6:8]),int(frontdate[8:10]) 
                    ymdstr             = frontdate[0:8]
                    hourstr            = frontdate[8:10]
                    
                    dst_date           = datetime.datetime(year,mon,day,hour)+ datetime.timedelta(hours = 1)
                    split_dsttime      = str(dst_date).split(' ')
                    ymd_dstlist        = str(split_dsttime[0]).replace('-','')  #得年月日字符串
                    time_dstlist       = str(split_dsttime[1]).split(':')
                    hour_dststr        = time_dstlist[0]
                    #最早时间加一小时，根据系统时间增加
                    frontdate          = ymd_dstlist + hour_dststr
                    
                    statymdpath        = os.path.join(statlogpath,ymdstr)
                    statfullpath       = os.path.join(statymdpath,hourstr)
                    if not os.path.exists(statfullpath):
#                   "The path you input is not having stat-yyyymmdd-hh path%s."%statfullpath
                        continue
                    else :
                        files = os.listdir(statfullpath)
#                       filenum = 0
                        for  file in files:
                            if not file.startswith('Stat_'):
                                continue
                            else:
#                               filenum += 1
                                fullfilename = os.path.join(statfullpath,file)
#                                print fullfilename
                            
                                fileInput = open(fullfilename, 'r')                
                                lines = fileInput.readlines()
                                #处理行
                                for line in lines :
                                    line = line[:-1]
                                    if len(check_keys) == 0:         #参数都为空，返回所以类型的记录
                                        dstlinelist.append(line)
                                    else:                            #有参数，则根据参数返回记录
                                        linelist = line.split('\t')
                                        flag = 0
                                        for check_key in check_keys:
                                            if check_key in linelist:
                                                flag = 1
                                            else :
                                                flag = 0
                                                break
                                        if flag == 1:
                                            dstlinelist.append(line)  
#                                       dstnum_suc += 1
#                       if filenum == 0 :
#                           print "%s has no Stat_****.log"%statfullpath
                except Exception, e:
                    print e
#           if dstnum_suc == 0 :
#               print "no line you need"
#               return  dstlinelist
   
        return  dstlinelist 
    except :
#   "Deal statinfo error."
        return  []


def     Handdeal(srcpath,dealdate):

    try:
        #检查输入目录是否存在
        if not os.path.exists(srcpath):
            print "The path %s is not existing"%srcpath
        else :
            #检查输入时间是否符合要求
            year,mon,day = int(dealdate[0:4]),int(dealdate[4:6]),int(dealdate[6:8])
            hour         = ''
            try:
                check_date  = datetime.datetime(year,mon,day)  
            except:
                print "The time you inputted is wrong"
                return
            
            deal_date       = datetime.datetime(year,mon,day)
            split_dsttime   = str(deal_date).split(' ')
            ymd_dstlist     = str(split_dsttime[0]).replace('-','')  #得年月日字符串
            time_dstlist    = str(split_dsttime[1]).split(':')
            
            #根据时间得到输入目录下需要的目录
            dealymdpath     = os.path.join(srcpath,ymd_dstlist)
            if not os.path.exists(dealymdpath):
                print "%s is not existing ."%dealymdpath
            else:
                for i in range(0,24):
                #按小时处理一天的所有数据   
                    if i < 10:
                        hour = '0' + str(i)
                    else :
                        hour = str(i)
                    dealfullpath    = os.path.join(dealymdpath,hour)
                    
                    try:
                        #判断小时目录是否存在
                        if not os.path.exists(dealfullpath):
                            continue
                        else :
                            #判断是否存在文件
                            files = os.listdir(dealfullpath)
                            if len(files) == 0:
                                continue
                            else:
                                gatherstrname = 'Stat_' + ymd_dstlist + hour + '.log'
                                gatherfilefullname = os.path.join(dealfullpath,gatherstrname)
                                if os.path.exists(gatherfilefullname) and os.stat(gatherfilefullname).st_size > 0 :              #汇总日志说明已经写过汇总日志
                                    continue 
                                else:
                                    #判断备份原始日志存不存在
#                                   print "start to write "
                                    formerfilenum = 0
                                    for file in files:
                                        if not file.startswith('FormerStat_'):
                                            continue
                                        else:
                    #写统计日志    
                                            formerfilenum += 1
                                            filefullname = os.path.join(dealfullpath,file)
                                            print filefullname
                                            WriteGatherlog(filefullname,gatherfilefullname)       
                                    if formerfilenum != 0 :
                                        print "[%s] write gatherlog success ." %dealfullpath 
                    except Exception, e:
                        print e 
    except Exception, e:
        print e      
        
                     
                        
def     WriteGatherlog(filefullname,gatherfilefullname):

#汇总日志固定格式
    Gatherkeyname = ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE', 'STATTIME', 'DATAFROM',\
                          'DATADEST', 'FILETYPE', 'SUCCFILENUM', 'SUCCFILESIZE', 'ERRFILENUM', 'ERRFILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM',\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE', 'STATPROGRAM']        
    try:       
        fileInput = open(filefullname, 'r')                
        lines = fileInput.readlines()
        gatherdstlines = []
        records_map    = {}


        for line in lines:
            try:
                line = line[:-1]
                Former_Map  = {}
           #将一条记录的信息以键对值读入Former_Map中,供写统计日志和写汇总日日志引用
           
                Former_Keys = line.split('\t')  #得到键值对列表
           
                for key in Former_Keys[1:]:
                    former_key = key.split(':')   #将键值对切分
                    formerbig_key = former_key[0].upper()
                
                    if formerbig_key in ['DEALTIME', 'FILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM' ,\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE']:
                        if former_key[1] == '':
				Former_Map[formerbig_key] = 0
			else:
				Former_Map[formerbig_key] = int(former_key[1])
                    else:
                        Former_Map[formerbig_key] = str(former_key[1])
            
            #检查记录行是否符合规范，主要检查FILENAME键值对是否存在和STATUS是否等于0或者-1。
                if  not Former_Map.has_key('FILETYPE')  :
                    print "[%s]formerlog line criterion error,no FILETYPE line: %s "%(filefullname,line)
                    continue
                if  not Former_Map.has_key('FILENAME')  :
                    print "[%s]formerlog line criterion error,no FILENAME line: %s "%(filefullname,line)
                    continue
                elif  not Former_Map.has_key('STATUS') :
                    print "[%s]formerlog line criterion error,no STATUS line: %s "%(filefullname,line)
                    continue
                elif Former_Map['FILENAME'] == '':
                    print "[%s]formerlog line criterion FILENAME error,line: %s "%(filefullname,line)
                    continue
                elif Former_Map['STATUS'] != '-1' and Former_Map['STATUS'] != '0':
                    print "[%s]formerlog line criterion STATUS error,line: %s "%(filefullname,line)
                    continue
                else:
#                   print " this line is right"
                    #汇总日志关键字，用来替换成值。形成一个值列表
                    Gatherkeys = ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE', 'STATTIME', 'DATAFROM',\
                          'DATADEST', 'FILETYPE', 'SUCCFILENUM', 'SUCCFILESIZE', 'ERRFILENUM', 'ERRFILESIZE', 'SUCCRECORDNUM', 'ERRRECORDNUM',\
                          'ENTRYFILENUM', 'ENTRYFILESIZE', 'SENDENTRYNUM', 'SENDENTRYSIZE', 'STATPROGRAM']
                    Gatherkeys[4] = int(time.time())     #更新合并后的最新操作时间STATTIME
                    if  Former_Map['STATUS'] == '0':    #处理错误的记录
                        Gatherkeys[8]  = 1
                        if Former_Map.has_key('FILESIZE'):
                            Gatherkeys[9]  = Former_Map['FILESIZE']
                        else :                           
                            Gatherkeys[9]  = 0
                        Gatherkeys[10] = 0
                        Gatherkeys[11] = 0
                    else :                                #处理正确的记录
                        Gatherkeys[8]  = 0
                        Gatherkeys[9]  = 0
                        Gatherkeys[10] = 1
                        if Former_Map.has_key('FILESIZE'):
                            Gatherkeys[11]  = Former_Map['FILESIZE']
                        else :
                            Gatherkeys[11]  = 0          
                    for i in range(0,len(Gatherkeys)):
                        #将统计日志Gatherkeys的其它关键字替换成值
                        if  Former_Map.has_key(Gatherkeys[i]):
                            Gatherkeys[i] = Former_Map[Gatherkeys[i]]
                        else :
                            if Gatherkeys[i] in ['SUCCRECORDNUM','ERRRECORDNUM' ,'ENTRYFILENUM','ENTRYFILESIZE','SENDENTRYNUM','SENDENTRYSIZE']:
                                Gatherkeys[i] = 0
                            elif Gatherkeys[i] in ['UPAREAID', 'PROTYPEDESC', 'DATASOURCE', 'DATADATE','DATAFROM','DATADEST','STATPROGRAM'] :
                                Gatherkeys[i] = ''                            
                    
                    checkstr = Gatherkeys[0] + Gatherkeys[1] + Gatherkeys[2] + Gatherkeys[3] + Gatherkeys[5] + Gatherkeys[6] + Gatherkeys[7] + Gatherkeys[18]
                
                    #对以上ckeckstr做MD5值校验，汇总信息           
                    m = md5instance()
                    m.update(checkstr)
                    md5_value = m.hexdigest()
                    if not records_map.has_key(md5_value):
                        records_map[md5_value] = Gatherkeys
                    else :
                        records_map[md5_value][4] = int(time.time())
                        for i in range(8,18):
                            records_map[md5_value][i] += Gatherkeys[i]
            except:
                print "[%s]formerlog line deal error,line: %s "%(filefullname,line)
        fileInput.close()
            
        #将汇总信息转成键值对形式的记录
        for md5_key in  records_map :
            gatherdstline = ''
            for i in range(0,len(Gatherkeyname)):
                gatherdstline = gatherdstline + str(Gatherkeyname[i]) + ":" +  str(records_map[md5_key][i]) + "\t" 
            gatherdstlines.append(gatherdstline[:-1] + '\n')
           
        #写汇总日志
        if len(gatherdstlines) > 0:
            try:
                fileOutput = open(gatherfilefullname,'a')
                fileOutput.writelines(gatherdstlines)
                fileOutput.close()
                print "[%s]write gatherlog  success. "%filefullname
            except:
                print "[%s]write gatherlog  error1."%filefullname
        else:            
            print "[%s]write gatherlog  error2. "%filefullname
   
        
    except:
        print "[%s]write gatherlog  error3. "%filefullname                                          
                  
            
   
def     readconf(stattaskinfolist):
    
    sectionlist    = []
    
    #读取配置文件
    conf = ConfigParser.ConfigParser()
    conf.read(CONF_PATH)
    sections = conf.sections()
    suclog   = FCLog(STAT_SUCCESS_LOG)
    errlog   = FCLog(STAT_FAIL_LOG)
    
    for section in sections:
        if section.startswith('TASK_'):   #匹配到的是任务节
            sectionlist.append(section)
        else:     #其它情况只有私有配置,添加配置内容
            g_bcpformat = conf.get(section,"STAT_BCPFORMAT")
            g_statdstpath = conf.get(section,"STAT_STATDST_PATH")
            g_stattmppath = conf.get(section,"STAT_STATTMP_PATH")
    
    #对任务节处理，添加任务信息。
    for tasksection in sectionlist:
        try:
            stattaskinfo = StatInfo()
            
            stattaskinfo.taskname          = tasksection
            stattaskinfo.statsrcpath       = conf.get(tasksection,"STAT_SRC_PATH")
            stattaskinfo.statwritemodule   = conf.get(tasksection,"STAT_WRITE_MODULE")
            stattaskinfo.bcpformat         = g_bcpformat
            stattaskinfo.statdstpath       = g_statdstpath
            stattaskinfo.stattmppath       = g_stattmppath
            stattaskinfo.suclog            = suclog
            stattaskinfo.errlog            = errlog
            
#           print stattaskinfo.statsrcpath
            #check the stat src dir 
            if not os.path.exists(stattaskinfo.statsrcpath) :
                stattaskinfo.errlog.error("[%s]the statsrcpath %s is not existing"%(tasksection,stattaskinfo.statsrcpath))
            else:
            #check the stat dst dir
                if not os.path.exists(stattaskinfo.statdstpath):
                    os.mkdir(stattaskinfo.statdstpath)
    
                if not os.path.exists(stattaskinfo.statt